#coding:utf-8
'''
* coder  : dzlua
* email  : 505544956@qq.com
* module : spider
* path   : jbdq
* file   : jbdq.py
* time   : 2017-11-13 13:59:17
'''
#--------------------#
import sys, codecs, os, time, re
from bs4 import BeautifulSoup
#----------#
sys.path.append("..")
#----------#
from spider import Spider
from database import DataBase
#--------------------#

#--------------------#
class Cpdq(Spider, DataBase):
    def __init__(self, host, user, pwd, db, threads_work = 4, tasks = 0):
        DataBase.__init__(self, host, user, pwd, db)
        Spider.__init__(self, threads_work, tasks)

        sql = "CREATE TABLE IF NOT EXISTS " + self.tb + \
            " (id INT AUTO_INCREMENT PRIMARY KEY," \
            " type TEXT," \
            " href TEXT," \
            " info TEXT )" \
            " DEFAULT CHARSET=utf8;"
        DataBase.execute(self, sql)
        #

        self.host = 'http://www.douguo.com/'
    def path(self, path):
        return os.path.join(self.host, path)
    def start(self):
        self.put(
            self.__on_page1,
            self.path('caipu') )
    def get_strings(self, soup):
        text = ''
        if not soup:
            return text
        for string in soup.strings:
            text += (string + '\n')
        return text
    def __on_page1(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        div = soup.find(id="container")
        if not div:
            self.log('__on_page1, no #container. %s' % url)
            return
        #
        eles = div.find_all('div', class_='cp_box')
        for ele in eles:
            img = ele.find('div', class_='cp_msg')
            if not img:
                continue
            a = img.find('a', class_='cp_name')
            if not a:
                continue
            info = {
                'type': a.text,
                'href': a.get('href')
            }
            self.put(self.__on_page2, info['href'], None, info)
        #
        page = soup.find('div', class_ = ['page_PN'])
        if not page:
            return
        for p in page.find_all('span', class_='floblock'):
            a = p.find('a')
            if a and a.text == u'下一页':
                self.put(self.__on_page1, a.get('href') )
    def __on_page2(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        table = soup.find('table', class_ = ['retamr'])
        text = self.get_strings(table)
        #
        step = soup.find('div', class_ = ['step clearfix'])
        text += self.get_strings(step)
        #
        xtieshi = soup.find('div', class_ = ['xtieshi'])
        text += self.get_strings(xtieshi)
        #
        info = {
            'type': args['type'],
            'href': url,
            'info': text,
        }
        self.save(self.__on_save, None, info)
    def __on_save(self, conn, data, args):
        print args['type'], args['href']

        sql = "INSERT INTO " + self.tb + \
            " (type, href, info)" \
            " VALUES(%s,%s,%s);"
        param = ( args['type'],args['href'],args['info'] )
        try:
            cursor = conn.cursor()
            cursor.execute(sql, param)
            cursor.close()
            conn.commit()
        except conn.Error, e:
            self.log('__on_save error. %d %s, %s ' % (e[0], e[1], args))
    #--------------------#
    def write_to_file(self, path):
        sql = 'SELECT * FROM %s;' % (self.tb)
        for r in self.select(sql):
            ff = os.path.join(path, r[1])
            ff += '.txt'
            content = self.del_empty_line(r[3])
            bok = self.create_file(ff, r[2] + '\n' + content)
            self.log('%s %s %s' % (r[0], bok, ff), not bok)
    def create_file(self, file, content=u''):
        if os.path.exists(file):
            return False
        path = os.path.dirname(file)
        self.mkdirs(path)
        #
        fp = codecs.open(file, 'w', 'utf-8')
        fp.write(content)
        fp.close()

        return True
    def mkdirs(self, path):
        try:
            os.makedirs(path)
        except:
            pass
    def del_empty_line(self, text):
        content = re.sub(r'\n[\s| ]*\n', '', text)
        return content
#--------------------#
